# _*_ coding:utf-8 _*_
# @Time  : 2022.08.11
# @Author: zizlee
import random
import re
import time
import json
import datetime
import requests
import pathlib
import pandas as pd
import numpy as np
from itertools import groupby
from urllib3 import disable_warnings
from zizlee_position import get_dominant_price_position, get_contract_price_position
disable_warnings()


USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) " \
             "Chrome/78.0.3904.108 Safari/537.36"


def get_ten_multiple(num: int):
    """ 获取大于num最近的整十数 """
    while num % 10 != 0:
        num += 1
    return num


def full_width_to_half_width(ustring):
    """ 全角转半角 """
    reverse_str = ""
    for uchar in ustring:
        inside_code = ord(uchar)
        if inside_code == 12288:  # 全角空格直接转换
            inside_code = 32
        elif 65281 <= inside_code <= 65374:  # 全角字符（除空格）根据关系转化
            inside_code -= 65248
        else:
            pass
        reverse_str += chr(inside_code)
    return reverse_str


def split_zh_en(ustring):
    """分离中英文"""
    zh_str = ""
    en_str = ""
    for uchar in ustring:
        inside_code = ord(uchar)
        if inside_code <= 256:
            en_str += uchar
        else:
            zh_str += uchar
    if not zh_str:
        zh_str = en_str
    return zh_str.strip(), en_str.strip()


def split_number_en(ustring):
    return [''.join(list(g)) for k, g in groupby(ustring, key=lambda x: x.isdigit())]


class DailyQuoteSpider(object):
    def __init__(self, date: datetime.datetime):
        self.quote_date = datetime.datetime.strptime(date.strftime('%Y-%m-%d'), '%Y-%m-%d')
        self.quote_date_string = self.quote_date.strftime('%Y%m%d')
        self.headers = {'User-Agent': USER_AGENT,
                        'Host': 'www.czce.com.cn'}
        self.quote_url = "http://www.czce.com.cn/cn/DFSStaticFiles/Future/{}/{}/FutureDataDaily.xls".format(
            self.quote_date.year, self.quote_date.strftime('%Y%m%d'))
        self.quote_file = pathlib.Path("czce/daily/{}.xls".format(self.quote_date_string))

        self.rank_url = "http://www.czce.com.cn/cn/DFSStaticFiles/Future/{}/{}/FutureDataHolding.xls".format(
            self.quote_date.year, self.quote_date.strftime('%Y%m%d'))
        self.rank_file = pathlib.Path("czce/rank/{}.xls".format(self.quote_date_string))

        self.receipt_url = "http://www.czce.com.cn/cn/DFSStaticFiles/Future/{}/{}/FutureDataWhsheet.xls".format(
            self.quote_date.year, self.quote_date.strftime('%Y%m%d'))
        self.receipt_file = pathlib.Path("czce/receipt/{}.xls".format(self.quote_date_string))

    # 获取日行情文件
    def get_quote_file(self):
        r = requests.get(self.quote_url, headers=self.headers)
        with open(self.quote_file, 'wb') as fp:
            fp.write(r.content)
        print('获取{}.CZC日行情数据完成！'.format(self.quote_date_string))

    # 获取日排名文件
    def get_rank_file(self):
        r = requests.get(self.rank_url, headers=self.headers)
        with open(self.rank_file, 'wb') as fp:
            fp.write(r.content)
        print('获取{}.CZC日排名数据完成！'.format(self.quote_date_string))

    # 获取日仓单文件
    def get_receipt_file(self):
        r = requests.get(self.receipt_url, headers=self.headers)
        with open(self.receipt_file, 'wb') as fp:
            fp.write(r.content)
        print('获取{}.CZC日仓单数据完成！'.format(self.quote_date_string))


class DailyQuoteParser(object):
    # SERVER_API = "http://127.0.0.1:8000/api/"
    SERVER_API = "https://210.13.218.130:9000/api/"

    def __init__(self, date: datetime.datetime):
        self.quote_date = datetime.datetime.strptime(date.strftime('%Y-%m-%d'), '%Y-%m-%d')
        self.quote_date_string = self.quote_date.strftime('%Y%m%d')

        self.quote_file = pathlib.Path("czce/daily/{}.xls".format(self.quote_date_string))
        self.rank_file = pathlib.Path("czce/rank/{}.xls".format(self.quote_date_string))
        self.receipt_file = pathlib.Path("czce/receipt/{}.xls".format(self.quote_date_string))
        self.delivery_file = pathlib.Path("czce/delivery/{}.xls".format(self.quote_date_string))

        self.quote_save_url = self.SERVER_API + "exchange/czce/daily/?date={}".format(self.quote_date.strftime("%Y-%m-%d"))
        self.rank_save_url = self.SERVER_API + "exchange/czce/rank/?date={}".format(self.quote_date.strftime("%Y-%m-%d"))
        self.receipt_save_url = self.SERVER_API + "exchange/czce/receipt/"

        self.resolution_quote_file = pathlib.Path("resolution/{}/CZC_Quote.json".format(self.quote_date_string))
        self.resolution_rank_file = pathlib.Path("resolution/{}/CZC_Rank.json".format(self.quote_date_string))
        self.resolution_receipt_file = pathlib.Path("resolution/{}/CZC_Receipt.json".format(self.quote_date_string))
        self.resolution_receipt_file2 = pathlib.Path("resolution/{}/CZC_Receipt2.json".format(self.quote_date_string))

        resolution_folder = pathlib.Path("resolution/{}/".format(self.quote_date_string))
        if not resolution_folder.exists():
            resolution_folder.mkdir(parents=True)

    # 将品种月份修改为品种+4位合约的形式
    @staticmethod
    def modify_contract_express(contract, current_date):
        date_str = current_date.strftime('%Y%m%d')
        split_contract = split_number_en(contract)
        contract = date_str[2].join(split_contract)  # 可能不是正确的合约年,需继续处理
        split_contract = split_number_en(contract)
        middle_contract = split_contract[1][:2]
        suffix_year = date_str[2:4]
        if int(middle_contract) < int(suffix_year):
            contract_year = '%02d' % get_ten_multiple(int(suffix_year))
            real_contract = split_contract[0] + '{}{}'.format(contract_year[0],
                                                              middle_contract[1]) + split_contract[1][-2:]
            return real_contract
        return contract

    # 解析日行情
    def parse_quote_file(self):
        if not self.quote_file.exists():
            print("没有发现{}.CZC的行情文件,请先抓取数据!".format(self.quote_date_string))
            return
        # 使用pandas解析数据
        xls_df = pd.read_excel(self.quote_file, thousands=',', skiprows=[0])
        # xls_df.columns = xls_df.iloc[0]  # 第一行作为列头
        # xls_df = xls_df.drop(xls_df.index[0])  # 删除第一行
        # xls_df = xls_df[~xls_df['品种月份'].str.contains('总计|小计')]  # 选取品种月份不含有小计和总计的行
        xls_df = xls_df[~xls_df['合约代码'].str.contains('总计|小计')]  # 合约代码  2021.04.14变化
        xls_df['品种代码'] = xls_df['合约代码'].apply(lambda x: x[:2])  # 变为品种
        # 2020-09-07 空盘量->持仓量
        if xls_df.columns.values.tolist() != [
            '合约代码', '昨结算', '今开盘', '最高价', '最低价', '今收盘', '今结算', '涨跌1', '涨跌2', '成交量(手)',
            '持仓量', '增减量', '成交额(万元)', '交割结算价', '品种代码'
        ]:
            print("{}.CZC日行情源数据文件格式有误,解析失败!".format(self.quote_date_string))
            return
        int_date = int(self.quote_date.timestamp())
        xls_df['日期'] = [int_date for _ in range(xls_df.shape[0])]
        xls_df = xls_df.fillna(0)
        xls_df.columns = [
            "contract", "pre_settlement", "open_price", "highest", "lowest", "close_price", "settlement",
            "zd_1", "zd_2",
            "trade_volume", "empty_volume", "increase_volume", "trade_price", "delivery_price", "variety_en", "date"
        ]
        # 将品种月份处理为4位合约(分离字母数字后插入年份的第3个数)
        xls_df["contract"] = xls_df["contract"].apply(self.modify_contract_express, args=(self.quote_date,))
        quote_data = xls_df.to_dict(orient='records')
        print('----- 解析{}.CZC行情文件成功,数量:{} -----'.format(self.quote_date_string, len(quote_data)))
        if len(quote_data) > 1:
            # 保存到本地文件
            with open(self.resolution_quote_file, 'w', encoding='utf8') as fp:
                json.dump(quote_data, fp, indent=2, ensure_ascii=False)

    # 解析日排名数据
    def parse_rank_file(self):
        if not self.rank_file.exists():
            print("没有发现{}.CZC的排名文件,请先抓取数据!".format(self.quote_date_string))
            return
        # 读取文件
        xls_df = pd.read_excel(self.rank_file, thousands=',')
        variety_index_dict = dict()  # 存品种的起始终止行
        contract_index_dict = dict()  # 存合约的起始终止行
        variety_dict = dict()  # 存品种的dict
        contract_set = set()  # 存合约的dict
        variety_en = None
        contract_en = None
        is_variety = True
        # 遍历每一行，取出每个品种的数据表
        # print(self.date)
        for row_content in xls_df.itertuples():
            # print(row_content[1])
            info_for_match_ = full_width_to_half_width(str(row_content[1]))
            search_variety = re.search(r'品种:(.*)\s日期.*', info_for_match_)  # 找到品种行开头
            search_contract = re.search(r'合约:(.*)\s日期.*', info_for_match_)  # 找到合约
            search_sum = re.search(r'合计', info_for_match_)
            if search_variety:  # 如果找到品种记录下品种和开始行
                zh_en_variety = search_variety.group(1)
                variety_name, variety_en = split_zh_en(zh_en_variety)
                if variety_en == "PTA":
                    variety_en = "TA"
                variety_dict[variety_en] = variety_name
                variety_index_dict[variety_en] = [row_content[0] + 1]
                is_variety = True
            elif search_contract:  # 如果找到合约,从品种数据去品种中文名,记录下开始行
                contract_en = search_contract.group(1)
                contract_set.add(contract_en)
                contract_index_dict[contract_en] = [row_content[0] + 1]
                is_variety = False
            else:
                pass  # 无则继续
            if search_sum and (variety_en or contract_en):  # 如果找到合计行，记录下当前品种的结束行
                if is_variety:

                    variety_index_dict[variety_en].append(row_content[0])
                else:
                    contract_index_dict[contract_en].append(row_content[0])
            else:
                pass  # 无则继续
        # 整理数据
        column_indexes = ['variety_en', 'contract', 'rank',
                          'trade_company', 'trade', 'trade_increase',
                          'long_position_company', 'long_position', 'long_position_increase',
                          'short_position_company', 'short_position', 'short_position_increase']

        result_df = pd.DataFrame(columns=column_indexes)
        # 每个品种数据框
        for variety_en in variety_dict:
            variety_index_range = variety_index_dict[variety_en]
            variety_df = xls_df.iloc[variety_index_range[0]:variety_index_range[1] + 1, :]
            variety_df = self._parser_rank_sub_df(variety_name=variety_dict[variety_en], sub_df=variety_df)
            # 填充品种代码和合约的值
            variety_df["variety_en"] = [variety_en for _ in range(variety_df.shape[0])]
            variety_df["contract"] = [variety_en for _ in range(variety_df.shape[0])]
            # print(variety_en, "\n", variety_df)
            result_df = pd.concat([result_df, variety_df])
        # 每个合约数据框
        for contract in contract_set:
            contract_index_range = contract_index_dict[contract]
            # variety_key = 'PTA' if contract[:2] == 'TA' else contract[:2]
            variety_key = contract[:2]
            # print(variety_dict[variety_key], contract, contract_index_range)
            contract_df = xls_df.iloc[contract_index_range[0]:contract_index_range[1] + 1, :]
            contract_df = self._parser_rank_sub_df(variety_name=variety_dict[variety_key], sub_df=contract_df)
            # 填充品种代码和合约的值
            contract_df["variety_en"] = [variety_key for _ in range(contract_df.shape[0])]
            target_contract = self.modify_contract_express(contract.strip(), self.quote_date)
            contract_df["contract"] = [target_contract for _ in range(contract_df.shape[0])]
            # print(contract, "\n", contract_df)
            result_df = pd.concat([result_df, contract_df])
        int_date = int(self.quote_date.timestamp())
        result_df["date"] = [int_date for _ in range(result_df.shape[0])]
        rank_data = result_df.to_dict(orient='records')
        print('----- 解析{}.CZC排名文件成功,数量:{} -----'.format(self.quote_date_string, len(rank_data)))
        if len(rank_data) > 1:
            # 保存到本地文件
            with open(self.resolution_rank_file, 'w', encoding='utf8') as fp:
                json.dump(rank_data, fp, indent=2, ensure_ascii=False)

    # 解析日仓单文件
    def parse_receipt_file(self):
        if not self.receipt_file.exists():
            print("没有发现{}.CZC的仓单文件,请先抓取数据!".format(self.quote_date_string))
            return
        # 读取文件
        xls_df = pd.read_excel(self.receipt_file)
        xls_df = xls_df.fillna('')
        variety_index_dict = dict()
        variety_dict = dict()
        variety_en = None  # 品种标记
        pre_variety_en = None
        for row_content in xls_df.itertuples():
            info_for_match_ = full_width_to_half_width(row_content[1])
            search_v = re.search(r'品种:(.*)\s单位.*', info_for_match_)  # 品种
            total_count = re.search(r'总计', info_for_match_)
            if search_v:  # 取得品种和品种的英文代码
                pre_variety_en = variety_en
                has_new_variety = True
                zh_en_variety = search_v.group(1)
                variety_name, variety_en = split_zh_en(zh_en_variety)
                if variety_en == "PTA":
                    variety_en = "TA"
                variety_dict[variety_en] = variety_name
                variety_index_dict[variety_en] = [row_content[0] + 1]
            else:
                has_new_variety = False
            # 获取当前品种的数据表
            if total_count and variety_en:
                variety_index_dict[variety_en].append(row_content[0])
            # 当没有总计时有上一个品种记录且找到了新品种，那么老品种结束行应该是找到新品种行的上一行
            if not total_count and pre_variety_en and has_new_variety:  # 补充没有总计时无法添加结束行的问题，该问题与20191111日后的数据出现
                variety_index_dict[pre_variety_en].append(row_content[0] - 1)

        # 整理数据
        data_list = []
        for variety_en in variety_dict:
            data_index_range = variety_index_dict[variety_en]  # 数据在dataFrame中的起终索引
            variety_df = xls_df.iloc[data_index_range[0]:data_index_range[1] + 1, :]
            # if variety_en == 'PK':
            #     print(variety_df)
            #     continue
            item = self._parser_receipt_sub_df(variety_en, variety_df)
            data_list.append(item)
        result_df = pd.DataFrame(data_list, columns=['variety_en', 'receipt', 'increase'])
        int_date = int(self.quote_date.timestamp())
        result_df["date"] = [int_date for _ in range(result_df.shape[0])]
        receipt_data = result_df.to_dict(orient='records')
        print('----- 解析{}.CZC仓单文件成功,数量:{} -----'.format(self.quote_date_string, len(receipt_data)))
        if len(receipt_data) > 1:
            # 保存到本地文件
            with open(self.resolution_receipt_file, 'w', encoding='utf8') as fp:
                json.dump(receipt_data, fp, indent=2, ensure_ascii=False)

    # 解析日仓单文件2(带仓库名的数据)
    def parse_receipt_file2(self):
        if not self.receipt_file.exists():
            print("没有发现{}.CZC的仓单文件,请先抓取数据!".format(self.quote_date_string))
            return
        # 执行文件分析
        xls_df = pd.read_excel(self.receipt_file)
        xls_df = xls_df.fillna('')
        variety_index_dict = dict()
        variety_dict = dict()
        variety_en = None  # 品种标记
        pre_variety_en = None
        for row_content in xls_df.itertuples():
            info_for_match_ = full_width_to_half_width(row_content[1])
            search_v = re.search(r'品种:(.*)\s单位.*', info_for_match_)  # 品种
            total_count = re.search(r'总计', info_for_match_)
            if search_v:  # 取得品种和品种的英文代码
                pre_variety_en = variety_en
                has_new_variety = True
                zh_en_variety = search_v.group(1)
                variety_name, variety_en = split_zh_en(zh_en_variety)
                if variety_en == "PTA":
                    variety_en = "TA"
                variety_dict[variety_en] = variety_name
                variety_index_dict[variety_en] = [row_content[0] + 1]
            else:
                has_new_variety = False
            # 获取当前品种的数据表
            if total_count and variety_en:
                variety_index_dict[variety_en].append(row_content[0])
            # 当没有总计时有上一个品种记录且找到了新品种，那么老品种结束行应该是找到新品种行的上一行
            if not total_count and pre_variety_en and has_new_variety:  # 补充没有总计时无法添加结束行的问题，该问题与20191111日后的数据出现
                variety_index_dict[pre_variety_en].append(row_content[0] - 1)

        # 整理数据
        column_indexes = ["warehouse", "variety_en", "receipt", "increase"]

        result_df = pd.DataFrame(columns=column_indexes)
        for variety_en in variety_dict:
            data_index_range = variety_index_dict[variety_en]  # 数据在dataFrame中的起终索引
            variety_df = xls_df.iloc[data_index_range[0]:data_index_range[1] + 1, :]
            variety_df = self._parser_receipt_sub_df(variety_en, variety_df, flag2=True)
            result_df = pd.concat([result_df, variety_df])
        # 增加ex_total字段
        result_df['ex_total'] = [0 for _ in range(result_df.shape[0])]
        # 增加日期字段df['receipt_date'] = [data_str for _ in range(df.shape[0])]
        result_df['receipt_date'] = self.quote_date.strftime('%Y-%m-%d')
        receipt_data = result_df.to_dict(orient='records')
        print('----- 解析{}.CZC仓单文件2成功,数量:{} -----'.format(self.quote_date_string, len(receipt_data)))
        if len(receipt_data) > 1:
            # 保存到本地文件
            with open(self.resolution_receipt_file2, 'w', encoding='utf8') as fp:
                json.dump(receipt_data, fp, indent=2, ensure_ascii=False)

    def _parser_rank_sub_df(self, variety_name, sub_df):
        """ 解析每个品种或合约的数据框 """
        column_indexes = sub_df.iloc[0].values.tolist()
        # print(column_indexes)
        if column_indexes != ['名次', '会员简称', '成交量（手）', '增减量', '会员简称', '持买仓量', '增减量', '会员简称', '持卖仓量', '增减量']:
            raise ValueError("{}郑商所的'{}'持仓排名数据表头格式有误!".format(self.date.strftime("%Y-%m-%d"), variety_name))
        column_indexes = ['名次', '成交会员', '成交量', '成交增减', '买仓会员', '买仓量', '买仓增减', '卖仓会员', '卖仓量', '卖仓增减']
        sub_df.columns = column_indexes
        # sub_df = sub_df.reindex(columns=column_indexes)  # 重新调整列
        sub_df = sub_df.drop(sub_df.index[0])  # 删除第一行
        # 去除合计行
        sub_df['名次'] = sub_df['名次'].apply(lambda x: str(x))
        sub_df = sub_df[~sub_df['名次'].str.contains('合计')]  # 选取不含有合计的行
        sub_df[['名次', '成交量', '成交增减', '买仓量', '买仓增减', '卖仓量', '卖仓增减']] = sub_df[['名次', '成交量', '成交增减', '买仓量', '买仓增减', '卖仓量', '卖仓增减']].replace('-', 0).astype(int)
        # 增加品种代码、合约两列
        new_column_indexes = ["品种代码", "合约"] + column_indexes

        sub_df = sub_df.reindex(columns=new_column_indexes)
        reset_indexes = ['variety_en', 'contract', 'rank',
                         'trade_company', 'trade', 'trade_increase',
                         'long_position_company', 'long_position', 'long_position_increase',
                         'short_position_company', 'short_position', 'short_position_increase']
        sub_df.columns = reset_indexes  # 修改列名
        return sub_df

    @staticmethod
    def _parser_receipt_sub_df(variety_en, variety_df, flag2=False):
        """ 解析每个品种的仓单日报 """
        # 20200220后的数据强筋小麦为机构简称和机构编号，仓单为‘确认书数量’
        # print(variety_df.iloc[0])
        variety_df.columns = variety_df.iloc[0].replace('机构编号', '仓库编号').replace('机构简称', '仓库简称').replace(
            '厂库编号', '仓库编号').replace('厂库简称', '仓库简称').replace('仓单数量(完税)', '仓单数量').replace('确认书数量', '仓单数量')  # 以第一行为列头
        # variety_df.columns = variety_df.iloc[0].replace('机构编号', '仓库编号').replace('机构简称', '仓库简称').replace(
        #     '厂库编号', '仓库编号').replace('厂库简称', '仓库简称').replace('仓单数量(完税)', '仓单数量').replace('确认书数量', '仓单数量')  # 以第一行为列头
        variety_df = variety_df.drop(variety_df.index[0])  # 删除第一行
        variety_df = variety_df[~variety_df['仓库编号'].str.contains('总计|小计')]  # 选取不含有小计和总计的行
        # 把仓库简称的列空置替换为NAN，并使用前一个进行填充
        variety_df['仓库编号'] = variety_df['仓库编号'].replace('', np.nan).fillna(method='ffill')
        variety_df['仓库简称'] = variety_df['仓库简称'].replace('', np.nan).fillna(method='ffill')
        variety_df['仓单数量'] = variety_df['仓单数量'].replace('', np.nan).fillna(0)
        # 将仓单数量列转为int
        variety_df['仓单数量'] = variety_df['仓单数量'].apply(lambda x: int(x))  # 转为int计算
        variety_df['当日增减'] = variety_df['当日增减'].apply(lambda x: int(x))
        if flag2:
            result_df = pd.DataFrame()
            result_df['仓单数量'] = variety_df['仓单数量'].groupby(variety_df['仓库简称']).sum()  # 计算和
            result_df['当日增减'] = variety_df['当日增减'].groupby(variety_df['仓库简称']).sum()
            result_df.reset_index()
            wh_name = variety_df[['仓库简称']].drop_duplicates(subset='仓库简称', keep='first')
            result_df = pd.merge(wh_name, result_df, on='仓库简称')
            result_df['品种代码'] = [variety_en for _ in range(result_df.shape[0])]
            # 将仓库简称的字符转为半角
            result_df["仓库简称"] = result_df["仓库简称"].apply(full_width_to_half_width)
            result_df = result_df.reindex(columns=["仓库简称", "品种代码", "仓单数量", "当日增减"])
            result_df.columns = ["warehouse", "variety_en", "receipt", "increase"]
            return result_df
        else:
            data = {'variety_en': variety_en, 'receipt': variety_df['仓单数量'].sum(), 'increase': variety_df['当日增减'].sum()}
            return data

    # 读取解析好的行情数据
    def read_daily_quote(self):
        if not self.resolution_quote_file.exists():
            raise ValueError('{}.CZC行情数据文件不存在,请先解析保存！'.format(self.quote_date_string))
        # 读取本地文件
        with open(self.resolution_quote_file, 'r', encoding='utf8') as fp:
            quote_data = json.load(fp)
        return quote_data

    # 读取解析好的排名数据
    def read_daily_rank(self):
        if not self.resolution_rank_file.exists():
            raise ValueError('{}.CZC排名数据文件不存在,请先解析保存！'.format(self.quote_date_string))
        # 读取本地文件
        with open(self.resolution_rank_file, 'r', encoding='utf8') as fp:
            rank_data = json.load(fp)
        return rank_data

    # 读取解析好的仓单数据
    def read_daily_receipt(self):
        if not self.resolution_rank_file.exists():
            raise ValueError('{}.CZC仓单数据文件不存在,请先解析保存！'.format(self.quote_date_string))
        # 读取本地文件
        with open(self.resolution_receipt_file, 'r', encoding='utf8') as fp:
            receipt_data = json.load(fp)
        return receipt_data

    # 读取解析好的仓单数据2
    def read_daily_receipt2(self):
        if not self.resolution_receipt_file2.exists():
            raise ValueError('{}.CZC仓单数据文件2不存在,请先解析保存！'.format(self.quote_date_string))
        # 读取本地文件
        with open(self.resolution_receipt_file2, 'r', encoding='utf8') as fp:
            receipt_data = json.load(fp)
        return receipt_data

    # 保存日行情数据
    def save_daily_quote(self):
        quote_data = self.read_daily_quote()
        try:
            r = requests.post(self.quote_save_url, json=quote_data, verify=False)
            print(r.json())
        except Exception as e:
            print('保存{}.CZC行情数据失败了:{}'.format(self.quote_date_string, e))
        time.sleep(1)

    # 保存日排名数据
    def save_daily_rank(self):
        rank_data = self.read_daily_rank()
        try:
            r = requests.post(self.rank_save_url, json=rank_data, verify=False)
            print(r.json())
        except Exception as e:
            print('保存{}.CZC排名数据失败了:{}'.format(self.quote_date_string, e))
        time.sleep(1)

    # 保存仓单数据
    def save_daily_receipt(self):
        receipt_data = self.read_daily_receipt()
        try:
            r = requests.post(self.receipt_save_url, json=receipt_data, verify=False)
            print(r.json())
        except Exception as e:
            print('保存{}.CZC仓单数据失败了:{}'.format(self.quote_date_string, e))
        time.sleep(1)

    # 总表接口:保存行情数据，所有交易所数据放一个表的
    def new_save_quote(self):
        quote_data = self.read_daily_quote()
        df = pd.DataFrame(quote_data)
        df['quotes_date'] = df['date'].apply(lambda x: datetime.datetime.fromtimestamp(x).strftime('%Y-%m-%d'))
        df.rename(columns={'empty_volume': 'position_volume'}, inplace=True)

        save_url = self.SERVER_API + 'dat/quotes/daily-quotes/'
        try:
            r = requests.post(save_url, json=df.to_dict(orient='records'), verify=False)
            r_data = r.json()
        except Exception as e:
            print('新版保存{}.CZC行情数据失败了:{}'.format(self.quote_date_string, e))
        else:
            print('新版保存{}.CZC行情数据成功:{},message:{}'.format(self.quote_date_string, r_data['count'], r_data['message']))
        time.sleep(1)

    # 总表接口:保存排名数据，所有交易所数据放一个表的
    def new_save_rank(self):
        rank_data = self.read_daily_rank()
        df = pd.DataFrame(rank_data)
        df['rank_date'] = df['date'].apply(lambda x: datetime.datetime.fromtimestamp(x).strftime('%Y-%m-%d'))
        save_url = self.SERVER_API + 'dat/rank/daily-rank/'
        try:
            r = requests.post(save_url, json=df.to_dict(orient='records'), verify=False)
            r_data = r.json()
        except Exception as e:
            print('新版保存{}.CZC排名数据失败了:{}'.format(self.quote_date_string, e))
        else:
            print(
                '新版保存{}.CZC排名数据成功:{},message:{}'.format(self.quote_date_string, r_data['count'], r_data['message']))
        time.sleep(1)

    # 总表接口:保存仓单数据，所有交易所数据放一个表的
    def new_save_receipt(self):
        receipt_data = self.read_daily_receipt2()
        df = pd.DataFrame(receipt_data)
        save_url = self.SERVER_API + 'dat/receipt/daily-receipt/'
        try:
            r = requests.post(save_url, json=df.to_dict(orient='records'), verify=False)
            r_data = r.json()
        except Exception as e:
            print('新版保存{}.CZC仓单数据失败了:{}'.format(self.quote_date_string, e))
        else:
            print(
                '新版保存{}.CZC仓单数据成功:{},message:{}'.format(self.quote_date_string, r_data['count'], r_data['message']))
        time.sleep(1)

    def run_positions_handler(self):
        # 日行情
        quote_data = self.read_daily_quote()
        quotes_df = pd.DataFrame(quote_data)
        quotes_df.rename(columns={'date': 'quotes_ts', 'empty_volume': 'position_volume'}, inplace=True)
        # 日排名
        rank_data = self.read_daily_rank()
        rank_df = pd.DataFrame(rank_data)
        rank_df.rename(columns={'date': 'rank_ts'}, inplace=True)
        if quotes_df.empty or rank_df.empty:
            print('{}.CZC行情或持仓数据为空!'.format(self.quote_date_string))
            return
        quotes_df = quotes_df[['quotes_ts', 'variety_en', 'contract', 'close_price', 'trade_volume', 'position_volume']]
        rank_df = rank_df[['rank_ts', 'variety_en', 'contract', 'long_position', 'short_position']]
        dominant_df = get_dominant_price_position(quotes_df.copy(), rank_df.copy())
        # 获取合约的持仓数据
        contract_df = get_contract_price_position(quotes_df.copy(), rank_df.copy())
        final_df = pd.concat([dominant_df, contract_df])
        for col in ['close_price', 'position_price', 'position_volume', 'long_position', 'short_position']:
            final_df[col] = final_df[col].apply(lambda x: int(x) if int(x) == float(x) else round(x, 4))
        final_df.sort_values(by='contract', inplace=True)
        # for row in final_df.to_dict(orient='records'):
        #     print(row)
        # 保存到服务器
        save_url = self.SERVER_API + 'dsas/price-position/'
        try:
            r = requests.post(save_url, json=final_df.to_dict(orient='records'), verify=False)
            r_data = r.json()
        except Exception as e:
            print('新版保存{}.CZC持仓价格数据失败了:{}'.format(self.quote_date_string, e))
        else:
            print('新版保存{}.CZC持仓价格数据成功:{}'.format(self.quote_date_string, r_data['message']))


if __name__ == '__main__':
    SPIDER = 0
    PARSER = 0
    delta_days = 0
    if SPIDER:
        daily_quote_spider = DailyQuoteSpider(date=datetime.datetime.today() + datetime.timedelta(days=delta_days))
        daily_quote_spider.get_quote_file()     # 获取日行情数据保存为文件
        daily_quote_spider.get_rank_file()      # 获取日排名数据保存为文件
        daily_quote_spider.get_receipt_file()   # 获取日仓单数据保存为文件
    else:
        parser = DailyQuoteParser(date=datetime.datetime.today() + datetime.timedelta(days=delta_days))
        if PARSER:
            parser.parse_quote_file()      # 解析日行情数据
            parser.parse_rank_file()       # 解析日排名数据
            parser.parse_receipt_file()    # 解析仓单数据
            parser.parse_receipt_file2()   # 解析、保存第二种仓单数据(含仓库简称)
        else:
            # --- 分库分表 ---
            parser.save_daily_quote()    # 保存日行情
            parser.save_daily_rank()     # 保存日排名
            parser.save_daily_receipt()  # 保存日仓单
            # --- 总表 ---
            parser.new_save_quote()      # 保存解析的行情数据
            parser.new_save_rank()       # 保存解析的持仓数据
            parser.new_save_receipt()    # 保存仓单的数据(含仓库简称)
            # -- 净持仓 ---
            parser.run_positions_handler()  # 处理保存净持仓数据



